# coding:utf-8
"""
    构造X，Y样本对，获取vocabs
"""
# 如果遇到导包问题
import sys

sys.path.append(r'D:\AI_workplace\GitProjects\mystudy_knowledge_graph\utils')
from data_process import *
from config import *

config = Config()


def build_data():
    """
    构造X,Y样本对，添加vocab
    :return:
    """
    datas = []
    sample_x = []
    sample_y = []
    vocab_list = ['PAD', 'UNK']
    with open(config.train_file_path, 'r', encoding='utf-8') as fr:
        for line in fr:
            word_tag_list = line.strip().split('\t')
            # print(word_tag_list)
            if not word_tag_list:
                continue
            word = word_tag_list[0]
            if not word:
                continue
            tag = word_tag_list[-1]
            sample_x.append(word)
            sample_y.append(tag)
            # 添加vocab
            if word not in vocab_list:
                vocab_list.append(word)
            # 如果遇到。！等符号，则将当前x，y添加进样本对，并清空
            if word in ['。', '!', '！', '?', '？']:
                datas.append([sample_x, sample_y])
                sample_x = []
                sample_y = []
    word2id = {word: index for index, word in enumerate(vocab_list)}
    with open(config.vocab_path, 'w', encoding='utf-8') as fw:
        fw.write('\n'.join(vocab_list))
    # print(len(datas))
    return datas, word2id


if __name__ == '__main__':
    build_data()
